//var citiesList = "ankang";
//
var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});
//var cities = citiesList.split(",");

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/Communities';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/Communities';
var async = require('async');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');

if (!String.prototype.format) {
    String.prototype.format = function () {
        var args = arguments;
        return this.replace(/{(\d+)}/g, function (match, number) {
            return typeof args[number] != 'undefined'
                ? args[number]
                : match
                ;
        });
    };
}
var filteredWords = '小区,区,街,路,市,县,苑,花苑,花园,园,属,院,楼,巷,公寓,公房,大厦,大楼,别墅'.split(',')
function filterAddress(addr) {
    var new_addr = addr
    filteredWords.forEach(function (word) {
        new_addr = new_addr.replace(word, '')
        //console.log(word)
    })
    return new_addr
}

var dbconn = null;

var cityLookup = {
    'aq': '安庆',
    'ay': '安阳',
    'ab': '阿坝藏族羌族自治州',
    'anshun': '安顺',
    'al': '阿里地',
    'ak': '安康',
    'aks': '阿克苏地',
    'altd': '阿勒泰地',
    'aomen': '澳门',
    'alsm': '阿拉善盟',
    'as': '鞍山',
    'bj': '北京',
    'bd': '保定',
    'bb': '蚌埠',
    'bozhou': '亳州',
    'bz': '滨州',
    'bh': '北海',
    'baise': '百色',
    'bazhong': '巴中',
    'bt': '包头',
    'bijie': '毕节地',
    'bs': '保山',
    'baoji': '宝鸡',
    'by': '白银',
    'bycem': '巴彦淖尔',
    'betl': '博尔塔拉蒙古自治州',
    'bygl': '巴音郭楞蒙古自治州',
    'bazhou': '巴州',
    'bx': '本溪',
    'baishan': '白山',
    'bc': '白城',
    'chengde': '承德',
    'chuzhou': '滁州',
    'cangzhou': '沧州',
    'ch': '巢湖',
    'chizhou': '池州',
    'changzhi': '长治',
    'cs': '长沙',
    'changde': '常德',
    'chenzhou': '郴州',
    'chaozhou': '潮州',
    'chongzuo': '崇左',
    'cq': '重庆',
    'cd': '成都',
    'cxyz': '楚雄彝族自治州',
    'chifeng': '赤峰',
    'changdu': '昌都地',
    'cjhz': '昌吉回族自治州',
    'changshu': '常熟',
    'cy': '朝阳',
    'cc': '长春',
    'cz': '常州',
    'dy': '东营',
    'dz': '德州',
    'dt': '大同',
    'dg': '东莞',
    'deyang': '德阳',
    'dazhou': '达州',
    'dlbz': '大理白族自治州',
    'dh': '德宏傣族景颇族自治州',
    'diqing': '迪庆藏族自治州',
    'dx': '定西',
    'dl': '大连',
    'dd': '丹东',
    'dq': '大庆',
    'dxal': '大兴安岭',
    'ez': '鄂州',
    'estjzmz': '恩施土家族苗族自治州',
    'erds': '鄂尔多斯',
    'fy': '阜阳',
    'fz': '福州',
    'fuzhou': '抚州',
    'fs': '佛山',
    'fcg': '防城港',
    'fushun': '抚顺',
    'fx': '阜新',
    'ganzhou': '赣州',
    'gz': '广州',
    'gl': '桂林',
    'guigang': '贵港',
    'guangyuan': '广元',
    'ga': '广安',
    'ganzi': '甘孜藏族自治州',
    'gy': '贵阳',
    'gnzz': '甘南藏族自治州',
    'guoluo': '果洛藏族自治州',
    'guyuan': '固原',
    'hd': '邯郸',
    'hn': '淮南',
    'huaibei': '淮北',
    'huangshan': '黄山',
    'hs': '衡水',
    'heze': '菏泽',
    'hb': '鹤壁',
    'hshi': '黄石',
    'hg': '黄冈',
    'hy': '衡阳',
    'hh': '怀化',
    'huizhou': '惠州',
    'heyuan': '河源',
    'hezhou': '贺州',
    'hc': '河池',
    'haikou': '海口',
    'hu': '呼和浩特',
    'honghe': '红河哈尼族彝族自治州',
    'hanzhong': '汉中',
    'hlbe': '呼伦贝尔',
    'haidong': '海东',
    'haibei': '海北藏族自治州',
    'huangnan': '黄南藏族自治州',
    'hnzz': '海南藏族自治州',
    'hx': '海西州',
    'hm': '哈密',
    'ht': '和田',
    'hld': '葫芦岛',
    'hrb': '哈尔滨',
    'hegang': '鹤岗',
    'heihe': '黑河',
    'ha': '淮安',
    'hz': '杭州',
    'huzhou': '湖州',
    'hf': '合肥',
    'jdz': '景德镇',
    'jj': '九江',
    'ja': '吉安',
    'jn': '济南',
    'jining': '济宁',
    'jiaozuo': '焦作',
    'jingmen': '荆门',
    'jingzhou': '荆州',
    'jincheng': '晋城',
    'jz': '晋中',
    'jm': '江门',
    'jy': '揭阳',
    'jyg': '嘉峪关',
    'jinchang': '金昌',
    'jq': '酒泉',
    'jiangyin': '江阴',
    'jinzhou': '锦州',
    'jl': '吉林',
    'jixi': '鸡西',
    'jms': '佳木斯',
    'jx': '嘉兴',
    'jh': '金华',
    'kaifeng': '开封',
    'km': '昆明',
    'klmy': '克拉玛依',
    'kzls': '克孜勒苏柯尔克孜自治州',
    'ksd': '喀什地区',
    'kunshan': '昆山',
    'la': '六安',
    'lf': '廊坊',
    'ly': '龙岩',
    'lw': '莱芜',
    'linyi': '临沂',
    'lc': '聊城',
    'luoyang': '洛阳',
    'luohe': '漯河',
    'ld': '娄底',
    'liuzhou': '柳州',
    'linfen': '临汾',
    'lb': '来宾',
    'luzhou': '泸州',
    'lvliang': '吕梁',
    'ls': '乐山',
    'liangshan': '凉山彝族自治州',
    'lps': '六盘水',
    'lj': '丽江',
    'lincang': '临沧',
    'lasa': '拉萨',
    'linzhi': '林芝',
    'lz': '兰州',
    'ln': '陇南',
    'lx': '临夏回族自治州',
    'liaoyang': '辽阳',
    'liaoyuan': '辽源',
    'lyg': '连云港',
    'lishui': '丽水',
    'mas': '马鞍山',
    'mm': '茂名',
    'mz': '梅州',
    'my': '绵阳',
    'ms': '眉山',
    'mdj': '牡丹江',
    'np': '南平',
    'nc': '南昌',
    'ny': '南阳',
    'nn': '南宁',
    'neijiang': '内江',
    'nanchong': '南充',
    'nujiang': '怒江傈僳族自治州',
    'nqd': '那曲地',
    'ningde': '宁德',
    'nj': '南京',
    'nt': '南通',
    'nb': '宁波',
    'pt': '莆田',
    'px': '萍乡',
    'pds': '平顶山',
    'puyang': '濮阳',
    'panzhihua': '攀枝花',
    'pl': '平凉',
    'pj': '盘锦',
    'qhd': '秦皇岛',
    'qz': '泉州',
    'qd': '青岛',
    'qingyuan': '清远',
    'qinzhou': '钦州',
    'qxn': '黔西南',
    'qdn': '黔东南',
    'qn': '黔南',
    'qj': '曲靖',
    'qingyang': '庆阳',
    'qqhr': '齐齐哈尔',
    'qth': '七台河',
    'quzhou': '衢州',
    'rizhao': '日照',
    'rkzd': '日喀则地',
    'sjz': '石家庄',
    'suzhou': '宿州',
    'sm': '三明',
    'sr': '上饶',
    'smx': '三门峡',
    'sq': '商丘',
    'shiyan': '十堰',
    'suizhou': '随州',
    'shaoyang': '邵阳',
    'shuozhou': '朔州',
    'sg': '韶关',
    'sz': '深圳',
    'st': '汕头',
    'sw': '汕尾',
    'sanya': '三亚',
    'suining': '遂宁',
    'simao': '思茅',
    'shannandi': '山南地',
    'sl': '商洛',
    'szs': '石嘴山',
    'shz': '石河子',
    'sy': '沈阳',
    'sp': '四平',
    'songyuan': '松原',
    'suihua': '绥化',
    'sh': '上海',
    'su': '苏州',
    'suqian': '宿迁',
    'sx': '绍兴',
    'tj': '天津',
    'ts': '唐山',
    'tongling': '铜陵',
    'ty': '太原',
    'ta': '泰安',
    'tr': '铜仁',
    'tongliao': '通辽',
    'tc': '铜川',
    'tianshui': '天水',
    'tlfd': '吐鲁番地',
    'tcd': '塔城地',
    'tw': '台湾',
    'taicang': '太仓',
    'tl': '铁岭',
    'th': '通化',
    'taizhou': '泰州',
    'tz': '台州',
    'wuhu': '芜湖',
    'wf': '潍坊',
    'weihai': '威海',
    'wh': '武汉',
    'wuzhou': '梧州',
    'wuhai': '乌海',
    'ws': '文山壮族苗族自治州',
    'wn': '渭南',
    'wuwei': '武威',
    'wuzhong': '吴忠',
    'wlmq': '乌鲁木齐',
    'wlcb': '乌兰察布',
    'wlcbm': '乌兰察布盟',
    'wx': '无锡',
    'wz': '温州',
    'wenchang': '文昌',
    'xt': '邢台',
    'xuancheng': '宣城',
    'xm': '厦门',
    'xinyu': '新余',
    'xx': '新乡',
    'xc': '许昌',
    'xy': '信阳',
    'xiaogan': '孝感',
    'xianning': '咸宁',
    'xiangtan': '湘潭',
    'xiangxi': '湘西土家族苗族自治州',
    'xinzhou': '忻州',
    'xa': '西安',
    'xianyang': '咸阳',
    'xn': '西宁',
    'xam': '兴安盟',
    'xg': '香港',
    'xiangyang': '襄阳',
    'xl': '锡林郭勒盟',
    'xsbn': '西双版纳',
    'xz': '徐州',
    'yingtan': '鹰潭',
    'yichun': '宜春',
    'yt': '烟台',
    'yq': '阳泉',
    'yc': '宜昌',
    'yy': '岳阳',
    'yiyang': '益阳',
    'yongzhou': '永州',
    'yuncheng': '运城',
    'yj': '阳江',
    'yf': '云浮',
    'yulin': '玉林',
    'yb': '宜宾',
    'ya': '雅安',
    'yx': '玉溪',
    'yanan': '延安',
    'yl': '榆林',
    'ys': '玉树藏族自治州',
    'yinchuan': '银川',
    'yili': '伊犁哈萨克自治州',
    'yiwu': '义乌',
    'yk': '营口',
    'yanbian': '延边朝鲜族自治州',
    'yashan': '鸭山',
    'yich': '伊春',
    'yancheng': '盐城',
    'yz': '扬州',
    'zjk': '张家口',
    'zhangzhou': '漳州',
    'zb': '淄博',
    'zaozhuang': '枣庄',
    'zz': '郑州',
    'zk': '周口',
    'zmd': '驻马店',
    'zhuzhou': '株洲',
    'zjj': '张家界',
    'zh': '珠海',
    'zhanjiang': '湛江',
    'zq': '肇庆',
    'zs': '中山',
    'zg': '自贡',
    'zy': '资阳',
    'zunyi': '遵义',
    'zt': '昭通',
    'zhangye': '张掖',
    'zw': '中卫',
    'zjw': '张家港',
    'zj': '镇江',
    'zhoushan': '舟山'
}

async.eachSeries(cities, function (city, callbackOuter) {
//cities.forEach(function (city) {
    var region = cityLookup[city]
    console.log("\n\n=============================%s STARTING=============================", city)
    var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=AvRqTglpaKICTZ5iwyn06KZj&page_size=1&page_num=0'
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=doqfhHvA5GjNLYnjjVFm9sL8&page_size=1&page_num=0'
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&address={1}&region={2}&tag=%E5%B0%8F%E5%8C%BA&scope=2&output=json&ak=ayGOiNeQNq6E9vSrbgeTrWbS&page_size=1&page_num=0'


    var dbCollection = 'xq_sfDetail_' + city;
    var collection =  null;

    var totalData = 0;
    var counter = 0;

    //var filters = {}
    //var filters = {'baidu': {'$in': [false, null]}}
    var filters = {$or:[{'lat':{$lte:20}},{'lat':{$gte:50}},{'lon':{$lte:100}},{'lon':{$gte:150}}]}

    function getExistingDataCount(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                dbconn = db
                assert.equal(null, err);
                //console.log("Connected correctly to server");
                collection = db.collection(dbCollection);
                collection.count(filters, function (err, count) {
                    totalData = count
                    console.log("Total data %s: %s", dbCollection, totalData)
                    if (totalData == 0)
                        callbackOuter()
                    else
                        callback()
                });
            });
    };

    function run() {
        var manifest = {
            target: {
                url: urlTemp,
                ranges: {
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            extractors: [{
                name: 'details',
                root: '',
                //root:'',
                fields: {}
            }]
        };

        function insertDocuments(doc, new_doc, callback) {
            var updateSpecs = {'baidu': true}
            //updateSpecs['city'] = region
            if ('name' in new_doc && filterAddress(doc.name) == filterAddress(new_doc.name)){
                if ('price' in new_doc && (doc.price == 0 || isNaN(doc.price))) {
                    updateSpecs['price'] = parseFloat(new_doc['price'])
                }
                if (Math.abs(new_doc.latitude - doc.lat) >= 1 || Math.abs(new_doc.longitude - doc.lon) >= 1) {
                    updateSpecs['lat'] = new_doc.latitude
                    updateSpecs['lon'] = new_doc.longitude
                    updateSpecs['baiduAddress'] = new_doc.address
                }
            }
            //if (Object.keys(updateSpecs).length > 0) {
            collection.updateOne({'_id': doc._id}, {$set: updateSpecs}, function (err, result) {
                if (!err) {
                    console.log("Updated %s (commId: %s) %s", doc.name, doc.commId, dbCollection);
                    console.log(updateSpecs)
                } else {
                    console.log(err)
                }
                callback();
            });
            //} else {
            //    console.log("No data for %s (commId: %s) %s", doc.name, doc.commId, dbCollection);
            //    callback();
            //}
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            //console.log('The job is done');
            //dbconn.close()
            // not exactly,
        });

        function init(callback) {
            //MongoClient.connect(connection,
            //    function (err, db) {
            //        assert.equal(null, err);
            //        //console.log("Connected correctly to server");
            //        dbconn = db;
            //        collection = db.collection(dbCollection)
                    callback(null, 'init');
            //    });
        }

        var ids = [];


        function findCommids(db, callback) {
            var collection = db.collection(dbCollection);
            collection.find(filters).toArray(
                function (err, docs) {

//            console.log(docs);
                    callback(docs);
                });
        };

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    ids = docs;
                    //docs.forEach(function(doc, i){
                    //    if(city == 'sh' && i > 23913){
                    //        ids.push(doc)
                    //    }
                    //})
                    //console.log(ids.length)
                    callback(null, 'getIds');
                }
            );
        }

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(ids, function (item, callbackInner) {
                    counter++
                    console.log("(%s of %s)", counter, totalData)

                    //var address = item.address.substr(item.address.indexOf(']') + 1)

                    var query = region + ' ' + item.name
                    var queryAddress = item.address
                    manifest.target.url = urlTemp.format(encodeURIComponent(query), encodeURIComponent(queryAddress), encodeURIComponent((region)))
                    //console.log(manifest.target.url)

                    var job1 = new Crawler(manifest);
                    //console.log(job1)

                    job1.on('data', function (data) {
                        //console.log(data)
                        var $ = cheerio.load(data.request.body);
                        var dataRoot = JSON.parse($._root.children[0].data)
                        if (parseInt(dataRoot.status) == 302) {
                            console.log("YOU HAVE BEEN BLOCKED BY BAIDU!")
                            process.exit()
                        }
                        else if (parseInt(dataRoot.status) == 240) {
                            console.log("INVALID DEVELOPER KEY!")
                            process.exit()
                        }
                        else if (parseInt(dataRoot.status) == 210) {
                            console.log("IP NOT ALLOWED FOR THIS DEVELOPER KEY!")
                            process.exit()
                        }

                        var new_doc = {}
                        try {
                            var data = dataRoot.results
                            var doc = data[0]  // only most relevant needed
                            for (var f in doc) {
                                if (f == 'location') {
                                    new_doc['longitude'] = doc[f]['lng']
                                    new_doc['latitude'] = doc[f]['lat']
                                } else if (f == 'detail_info') {
                                    for (var df in doc[f]) {
                                        new_doc[df] = doc[f][df]
                                    }
                                } else {
                                    new_doc[f] = doc[f]
                                }
                            }
                        } catch (e) {
                            console.log(e)
                        }
                        insertDocuments(item, new_doc, function () {
                            //callback()
                            if (counter == totalData) {
                                //dbconn.close()
                                //callback()
                                callbackOuter()
                            }
                        })

                    });

                    job1.on('error', function (err) {
                        throw err;
                    });

                    job1.on('end', function () {

                        console.log('The job is done: %d', ++cnt);
                        delete job1;

                        callbackInner();

                    });

                    job1.start();

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            //console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                //console.log(results);
            });
    }

    getExistingDataCount(run);
})
;
